package com.curry.apiprovider.crawler.pageProcessor.job;

import com.curry.apiprovider.crawler.Constants;
import com.curry.apiprovider.util.StringUtils;
import lombok.AllArgsConstructor;
import lombok.Data;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 前程无忧的爬虫
 */
public class FirstPageProcessor implements PageProcessor {

    private static final Site site = Site.me().setRetryTimes(3).setCharset("GB2312").setSleepTime(1000);
    private String params[];

    public FirstPageProcessor(String[] params) {
        this.params = params;
    }


    @Override
    public void process(Page page) {
        String size_str = page.getHtml().css("div.dw_table").css("div.dw_page").css("div.p_in").xpath("//span[@class='td']/text()").get();
        String regEx = "[^0-9]";
        Pattern p = Pattern.compile(regEx);
        Matcher m = p.matcher(size_str);
        int totalPage = Integer.valueOf(m.replaceAll("").trim());
        page.putField(Constants.KEY_JOB_LIST_PAGE_URL, String.valueOf(totalPage));
    }

    @Override
    public Site getSite() {
        return site;
    }
}
